# -*- coding: utf-8 -*-
u"""用来爬取DIANA数据库中miRGen v.3版本库."""
import re
import scrapy
from scrapy.http import Request
from bs4 import BeautifulSoup
from scripts.diana import ExtractMirna
from diana.items import DianaItem
# from termcolor import cprint


class MirgenSpider(scrapy.Spider):
    u"""爬虫."""
    name = "miRGen"
    allowed_domains = [
        "http://carolina.imis.athena-innovation.gr/diana_tools/web/index.php"]
    start_urls = [
        'http://carolina.imis.athena-innovation.gr/diana_tools/web/index.php/']

    start_urls.extend(
        ['?r=mirgenv3%2Findex&precursors%5B%5D=', None,
         '&tfs%5B%5D=&tissueCell=0&upstream=3000&downstream=1000&pvalue=5.0E-5&tpm=1&filters=0'])

    def start_requests(self):
        u"""开始爬取."""
        mirnas = ExtractMirna().mirna_names

        for i in mirnas:
            self.start_urls[2] = i
            url = ''.join(self.start_urls)
            # cprint(url, 'red')
            yield Request(url, self.parse)

    def parse(self, response):
        u"""测试."""
        items = []
        # print(response.text)
        soup = BeautifulSoup(response.text, 'lxml')
        containers = soup.findAll("div", {"class": "first-level-container"})

        if len(containers) >= 1:
            for container in containers:
                msgs = container.findAll(
                    "div", {"class": re.compile('first-level-block')})

                msg = [x.get_text() for x in msgs]
                item = DianaItem()
                item['mirna_name'] = msg[0]
                item['tss'] = msg[1]
                item['tissue'] = msg[2]
                items.append(item)
        return items
